{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "4                           Allen, Mr. William Henry    male  35.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  \n",
       "4      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas #ipython notebook\n",
    "titanic = pandas.read_csv(\"titanic_train.csv\")\n",
    "titanic.head(5)\n",
    "#print (titanic.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       PassengerId    Survived      Pclass         Age       SibSp  \\\n",
      "count   891.000000  891.000000  891.000000  891.000000  891.000000   \n",
      "mean    446.000000    0.383838    2.308642   29.361582    0.523008   \n",
      "std     257.353842    0.486592    0.836071   13.019697    1.102743   \n",
      "min       1.000000    0.000000    1.000000    0.420000    0.000000   \n",
      "25%     223.500000    0.000000    2.000000   22.000000    0.000000   \n",
      "50%     446.000000    0.000000    3.000000   28.000000    0.000000   \n",
      "75%     668.500000    1.000000    3.000000   35.000000    1.000000   \n",
      "max     891.000000    1.000000    3.000000   80.000000    8.000000   \n",
      "\n",
      "            Parch        Fare  \n",
      "count  891.000000  891.000000  \n",
      "mean     0.381594   32.204208  \n",
      "std      0.806057   49.693429  \n",
      "min      0.000000    0.000000  \n",
      "25%      0.000000    7.910400  \n",
      "50%      0.000000   14.454200  \n",
      "75%      0.000000   31.000000  \n",
      "max      6.000000  512.329200  \n"
     ]
    }
   ],
   "source": [
    "titanic[\"Age\"] = titanic[\"Age\"].fillna(titanic[\"Age\"].median())\n",
    "print titanic.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['male' 'female']\n"
     ]
    }
   ],
   "source": [
    "print titanic[\"Sex\"].unique()\n",
    "\n",
    "# Replace all the occurences of male with the number 0.\n",
    "titanic.loc[titanic[\"Sex\"] == \"male\", \"Sex\"] = 0\n",
    "titanic.loc[titanic[\"Sex\"] == \"female\", \"Sex\"] = 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['S' 'C' 'Q' nan]\n"
     ]
    }
   ],
   "source": [
    "print titanic[\"Embarked\"].unique()\n",
    "titanic[\"Embarked\"] = titanic[\"Embarked\"].fillna('S')\n",
    "titanic.loc[titanic[\"Embarked\"] == \"S\", \"Embarked\"] = 0\n",
    "titanic.loc[titanic[\"Embarked\"] == \"C\", \"Embarked\"] = 1\n",
    "titanic.loc[titanic[\"Embarked\"] == \"Q\", \"Embarked\"] = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Import the linear regression class\n",
    "from sklearn.linear_model import LinearRegression\n",
    "# Sklearn also has a helper that makes it easy to do cross validation\n",
    "from sklearn.cross_validation import KFold\n",
    "\n",
    "# The columns we'll use to predict the target\n",
    "predictors = [\"Pclass\", \"Sex\", \"Age\", \"SibSp\", \"Parch\", \"Fare\", \"Embarked\"]\n",
    "\n",
    "# Initialize our algorithm class\n",
    "alg = LinearRegression()\n",
    "# Generate cross validation folds for the titanic dataset.  It return the row indices corresponding to train and test.\n",
    "# We set random_state to ensure we get the same splits every time we run this.\n",
    "kf = KFold(titanic.shape[0], n_folds=3, random_state=1)\n",
    "\n",
    "predictions = []\n",
    "for train, test in kf:\n",
    "    # The predictors we're using the train the algorithm.  Note how we only take the rows in the train folds.\n",
    "    train_predictors = (titanic[predictors].iloc[train,:])\n",
    "    # The target we're using to train the algorithm.\n",
    "    train_target = titanic[\"Survived\"].iloc[train]\n",
    "    # Training the algorithm using the predictors and target.\n",
    "    alg.fit(train_predictors, train_target)\n",
    "    # We can now make predictions on the test fold\n",
    "    test_predictions = alg.predict(titanic[predictors].iloc[test,:])\n",
    "    predictions.append(test_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.783389450056\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "e:\\python27\\lib\\site-packages\\ipykernel\\__main__.py:10: FutureWarning: in the future, boolean array-likes will be handled as a boolean array index\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# The predictions are in three separate numpy arrays.  Concatenate them into one.  \n",
    "# We concatenate them on axis 0, as they only have one axis.\n",
    "predictions = np.concatenate(predictions, axis=0)\n",
    "\n",
    "# Map predictions to outcomes (only possible outcomes are 1 and 0)\n",
    "predictions[predictions > .5] = 1\n",
    "predictions[predictions <=.5] = 0\n",
    "accuracy = sum(predictions[predictions == titanic[\"Survived\"]]) / len(predictions)\n",
    "print accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.787878787879\n"
     ]
    }
   ],
   "source": [
    "from sklearn import cross_validation\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "# Initialize our algorithm\n",
    "alg = LogisticRegression(random_state=1)\n",
    "# Compute the accuracy score for all the cross validation folds.  (much simpler than what we did before!)\n",
    "scores = cross_validation.cross_val_score(alg, titanic[predictors], titanic[\"Survived\"], cv=3)\n",
    "# Take the mean of the scores (because we have one for each fold)\n",
    "print(scores.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "titanic_test = pandas.read_csv(\"test.csv\")\n",
    "titanic_test[\"Age\"] = titanic_test[\"Age\"].fillna(titanic[\"Age\"].median())\n",
    "titanic_test[\"Fare\"] = titanic_test[\"Fare\"].fillna(titanic_test[\"Fare\"].median())\n",
    "titanic_test.loc[titanic_test[\"Sex\"] == \"male\", \"Sex\"] = 0 \n",
    "titanic_test.loc[titanic_test[\"Sex\"] == \"female\", \"Sex\"] = 1\n",
    "titanic_test[\"Embarked\"] = titanic_test[\"Embarked\"].fillna(\"S\")\n",
    "\n",
    "titanic_test.loc[titanic_test[\"Embarked\"] == \"S\", \"Embarked\"] = 0\n",
    "titanic_test.loc[titanic_test[\"Embarked\"] == \"C\", \"Embarked\"] = 1\n",
    "titanic_test.loc[titanic_test[\"Embarked\"] == \"Q\", \"Embarked\"] = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.785634118967\n"
     ]
    }
   ],
   "source": [
    "from sklearn import cross_validation\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "predictors = [\"Pclass\", \"Sex\", \"Age\", \"SibSp\", \"Parch\", \"Fare\", \"Embarked\"]\n",
    "\n",
    "# Initialize our algorithm with the default paramters\n",
    "# n_estimators is the number of trees we want to make\n",
    "# min_samples_split is the minimum number of rows we need to make a split\n",
    "# min_samples_leaf is the minimum number of samples we can have at the place where a tree branch ends (the bottom points of the tree)\n",
    "alg = RandomForestClassifier(random_state=1, n_estimators=10, min_samples_split=2, min_samples_leaf=1)\n",
    "# Compute the accuracy score for all the cross validation folds.  (much simpler than what we did before!)\n",
    "kf = cross_validation.KFold(titanic.shape[0], n_folds=3, random_state=1)\n",
    "scores = cross_validation.cross_val_score(alg, titanic[predictors], titanic[\"Survived\"], cv=kf)\n",
    "\n",
    "# Take the mean of the scores (because we have one for each fold)\n",
    "print(scores.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.814814814815\n"
     ]
    }
   ],
   "source": [
    "alg = RandomForestClassifier(random_state=1, n_estimators=100, min_samples_split=4, min_samples_leaf=2)\n",
    "# Compute the accuracy score for all the cross validation folds.  (much simpler than what we did before!)\n",
    "kf = cross_validation.KFold(titanic.shape[0], 3, random_state=1)\n",
    "scores = cross_validation.cross_val_score(alg, titanic[predictors], titanic[\"Survived\"], cv=kf)\n",
    "\n",
    "# Take the mean of the scores (because we have one for each fold)\n",
    "print(scores.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Generating a familysize column\n",
    "titanic[\"FamilySize\"] = titanic[\"SibSp\"] + titanic[\"Parch\"]\n",
    "\n",
    "# The .apply method generates a new series\n",
    "titanic[\"NameLength\"] = titanic[\"Name\"].apply(lambda x: len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mr          517\n",
      "Miss        182\n",
      "Mrs         125\n",
      "Master       40\n",
      "Dr            7\n",
      "Rev           6\n",
      "Col           2\n",
      "Major         2\n",
      "Mlle          2\n",
      "Countess      1\n",
      "Ms            1\n",
      "Lady          1\n",
      "Jonkheer      1\n",
      "Don           1\n",
      "Mme           1\n",
      "Capt          1\n",
      "Sir           1\n",
      "Name: Name, dtype: int64\n",
      "1     517\n",
      "2     183\n",
      "3     125\n",
      "4      40\n",
      "5       7\n",
      "6       6\n",
      "7       5\n",
      "10      3\n",
      "8       3\n",
      "9       2\n",
      "Name: Name, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "# A function to get the title from a name.\n",
    "def get_title(name):\n",
    "    # Use a regular expression to search for a title.  Titles always consist of capital and lowercase letters, and end with a period.\n",
    "    title_search = re.search(' ([A-Za-z]+)\\.', name)\n",
    "    # If the title exists, extract and return it.\n",
    "    if title_search:\n",
    "        return title_search.group(1)\n",
    "    return \"\"\n",
    "\n",
    "# Get all the titles and print how often each one occurs.\n",
    "titles = titanic[\"Name\"].apply(get_title)\n",
    "print(pandas.value_counts(titles))\n",
    "\n",
    "# Map each title to an integer.  Some titles are very rare, and are compressed into the same codes as other titles.\n",
    "title_mapping = {\"Mr\": 1, \"Miss\": 2, \"Mrs\": 3, \"Master\": 4, \"Dr\": 5, \"Rev\": 6, \"Major\": 7, \"Col\": 7, \"Mlle\": 8, \"Mme\": 8, \"Don\": 9, \"Lady\": 10, \"Countess\": 10, \"Jonkheer\": 10, \"Sir\": 9, \"Capt\": 7, \"Ms\": 2}\n",
    "for k,v in title_mapping.items():\n",
    "    titles[titles == k] = v\n",
    "\n",
    "# Verify that we converted everything.\n",
    "print(pandas.value_counts(titles))\n",
    "\n",
    "# Add in the title column.\n",
    "titanic[\"Title\"] = titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfwAAAGvCAYAAAC+SGdKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XuYXVV9//H3V5BiQIKCBKxGUS6mXtBERVTUFgXRYsH2\nR52KWBFRK4r5ScFalQBeoRqU0sqvtfWCjqLV4gVFbip4QSV4ocYoljhyi45gUCCi8P39sfaQM5PJ\nZM7Mmdn7nP1+Pc95OGftfWa+5DlnPnuvvfZakZlIkqTBdq+6C5AkSXPPwJckqQUMfEmSWsDAlySp\nBQx8SZJawMCXJKkFDHxJklrAwJckqQUMfEmSWsDAlySpBboK/Ii4NiLunuRxZsc+p0TEDRFxe0Rc\nGBF79L5sSZLUjW7P8B8P7NrxeBaQwLkAEXEicCxwDPBE4DbggojYplcFS5Kk7sVsFs+JiDOA52Tm\nXtXrG4DTM3Nl9XoHYB3w4sw8twf1SpKkGZjxNfyIuDfwQuD91evdKWf9F4/tk5m3AlcA+82uTEmS\nNBtbz+K9hwELgQ9Wr3eldO+vm7DfumrbpCJiJ+AgYC2wYRb1SJLUNtsCDwUuyMxfTbXjbAL/KOAL\nmXnTLH4GlLD/yCx/hiRJbfZC4KNT7TCjwI+IxcAzgUM7mm8CAljE+LP8RcBVU/y4tQDnnHMOS5Ys\nmUk5tVq+fDkrV66su4yu9Wvd0L+192vd0L+192vd0L+192vd0J+1r169miOOOAKqLJ3KTM/wj6KE\n+vljDZl5bUTcBBwAfB/uGbS3L3DWFD9rA8CSJUtYunTpDMupz8KFC617nvVr7f1aN/Rv7f1aN/Rv\n7f1aN/R37UzjknjXgR8RAfwt8IHMvHvC5jOAN0bENZSjjVOB64Dzuv09kiSpd2Zyhv9M4MHAf07c\nkJmnRcQC4GxgR+Ay4ODMvHNWVTbYHXfcwapVq+ouY5ydd96ZxYsX112GJKlBug78zLwQ2GqK7SuA\nFTMvqX+MjIxwySWXsmzZsrpLGWfbbRewZs1qQ1+SdI/ZjNJvvdHRUe6++y7gHKApAw5Xs2HDEYyO\njk4Z+ENDQ/NYU2/1a+39Wjf0b+39Wjf0b+39Wjf0d+3TMauZ9npSQMRS4Morr7yy7wZLrFq1qjq7\nvxJoSu2rgGX047+nJKk7G3OIZZk55fVlV8uTJKkFDHxJklrAwJckqQUMfEmSWsDAlySpBQx8SZJa\nwMCXJKkFDHxJklrAwJckqQUMfEmSWsDAlySpBQx8SZJawMCXJKkFDHxJklrAwJckqQUMfEmSWsDA\nlySpBQx8SZJawMCXJKkFDHxJklrAwJckqQUMfEmSWsDAlySpBQx8SZJawMCXJKkFDHxJklrAwJck\nqQUMfEmSWsDAlySpBQx8SZJawMCXJKkFug78iHhgRHw4IkYj4vaI+F5ELJ2wzykRcUO1/cKI2KN3\nJUuSpG51FfgRsSPwNeB3wEHAEuB1wC0d+5wIHAscAzwRuA24ICK26VHNkiSpS1t3uf/rgZHMPLqj\n7WcT9jkOODUzPwcQEUcC64BDgXNnWqgkSZq5brv0DwG+ExHnRsS6iFgVEfeEf0TsDuwKXDzWlpm3\nAlcA+/WiYEmS1L1uA/9hwCuBNcCBwL8C742IF1XbdwWSckbfaV21TZIk1aDbLv17Ad/KzDdVr78X\nEY8CXgF8uKeVSZKknuk28G8EVk9oWw08v3p+ExDAIsaf5S8CrprqBy9fvpyFCxeOaxsaGmJoaKjL\nEiVJGjzDw8MMDw+Pa1u/fv20399t4H8N2HtC295UA/cy89qIuAk4APg+QETsAOwLnDXVD165ciVL\nly6dahdJklprspPgVatWsWzZsmm9v9vAXwl8LSL+gTLifl/gaOBlHfucAbwxIq4B1gKnAtcB53X5\nuyRJUo90FfiZ+Z2IOAx4B/Am4FrguMz8WMc+p0XEAuBsYEfgMuDgzLyzd2VLkqRudHuGT2aeD5y/\nhX1WACtmVpIkSeo159KXJKkFDHxJklrAwJckqQUMfEmSWsDAlySpBQx8SZJawMCXJKkFDHxJklrA\nwJckqQUMfEmSWsDAlySpBQx8SZJawMCXJKkFDHxJklrAwJckqQUMfEmSWsDAlySpBQx8SZJawMCX\nJKkFDHxJklrAwJckqQUMfEmSWsDAlySpBQx8SZJawMCXJKkFDHxJklrAwJckqQUMfEmSWsDAlySp\nBQx8SZJawMCXJKkFDHxJklrAwJckqQUMfEmSWqCrwI+IkyLi7gmPH07Y55SIuCEibo+ICyNij96W\nLEmSujWTM/yrgUXArtXjqWMbIuJE4FjgGOCJwG3ABRGxzexLlSRJM7X1DN7zh8z85Wa2HQecmpmf\nA4iII4F1wKHAuTMrUZIkzdZMzvD3jIjrI+KnEXFORDwYICJ2p5zxXzy2Y2beClwB7NeTaiVJ0ox0\nG/jfBP4WOAh4BbA78NWI2I4S9kk5o++0rtomSZJq0lWXfmZe0PHy6oj4FvAz4HDgR7MpZPny5Sxc\nuHBc29DQEENDQ7P5sZIkDYTh4WGGh4fHta1fv37a75/JNfx7ZOb6iPgxsAfwZSAoA/o6z/IXAVdt\n6WetXLmSpUuXzqYcSZIG1mQnwatWrWLZsmXTev+s7sOPiO0pYX9DZl4L3AQc0LF9B2Bf4Ouz+T2S\nJGl2ujrDj4jTgc9SuvH/GDgZ+D3wsWqXM4A3RsQ1wFrgVOA64Lwe1StJkmag2y79BwEfBXYCfglc\nDjwpM38FkJmnRcQC4GxgR+Ay4ODMvLN3JUuSpG51O2hviyPoMnMFsGKG9UiSpDngXPqSJLWAgS9J\nUgsY+JIktYCBL0lSCxj4kiS1gIEvSVILGPiSJLWAgS9JUgsY+JIktYCBL0lSCxj4kiS1gIEvSVIL\nGPiSJLWAgS9JUgsY+JIktYCBL0lSCxj4kiS1gIEvSVILGPiSJLWAgS9JUgsY+JIktYCBL0lSCxj4\nkiS1gIEvSVILGPiSJLWAgS9JUgsY+JIktYCBL0lSCxj4kiS1gIEvSVILGPiSJLWAgS9JUgsY+JIk\ntcCsAj8iXh8Rd0fEuye0nxIRN0TE7RFxYUTsMbsyJUnSbMw48CPiCcAxwPcmtJ8IHFtteyJwG3BB\nRGwzizolSdIszCjwI2J74BzgaODXEzYfB5yamZ/LzKuBI4EHAofOplBJkjRzMz3DPwv4bGZe0tkY\nEbsDuwIXj7Vl5q3AFcB+My1SkiTNztbdviEiXgA8Fnj8JJt3BRJYN6F9XbVNkiTVoKvAj4gHAWcA\nz8zM3/eykOXLl7Nw4cJxbUNDQwwNDfXy10iS1JeGh4cZHh4e17Z+/fppv7/bM/xlwAOAVRERVdtW\nwNMi4ljgEUAAixh/lr8IuGqqH7xy5UqWLl3aZTmSJLXDZCfBq1atYtmyZdN6f7fX8C8CHk3p0t+n\nenyHMoBvn8z8X+Am4ICxN0TEDsC+wNe7/F2SJKlHujrDz8zbgB92tkXEbcCvMnN11XQG8MaIuAZY\nC5wKXAecN+tqJUnSjHQ9aG8SOe5F5mkRsQA4G9gRuAw4ODPv7MHvkiRJMzDrwM/MP5ukbQWwYrY/\nW5Ik9YZz6UuS1AIGviRJLWDgS5LUAga+JEktYOBLktQCBr4kSS1g4EuS1AIGviRJLWDgS5LUAga+\nJEktYOBLktQCBr4kSS1g4EuS1AIGviRJLWDgS5LUAga+JEktYOBLktQCBr4kSS1g4EuS1AIGviRJ\nLWDgS5LUAga+JEktYOBLktQCBr4kSS1g4EuS1AIGviRJLWDgS5LUAga+JEktYOBLktQCBr4kSS1g\n4EuS1AIGviRJLWDgS5LUAl0FfkS8IiK+FxHrq8fXI+LZE/Y5JSJuiIjbI+LCiNijtyVLkqRudXuG\n/3PgRGApsAy4BDgvIpYARMSJwLHAMcATgduACyJim55VLEmSutZV4Gfm5zPzi5n508y8JjPfCPwW\neFK1y3HAqZn5ucy8GjgSeCBwaE+rliRJXZnxNfyIuFdEvABYAHw9InYHdgUuHtsnM28FrgD2m22h\nkiRp5rbu9g0R8SjgG8C2wG+AwzJzTUTsBySwbsJb1lEOBCRJUk26DnzgR8A+wELgr4APRcTTZlvI\n8uXLWbhw4bi2oaEhhoaGZvujJUnqe8PDwwwPD49rW79+/bTf33XgZ+YfgP+tXl4VEU+kXLs/DQhg\nEePP8hcBV23p565cuZKlS5d2W44kSa0w2UnwqlWrWLZs2bTe34v78O8F/FFmXgvcBBwwtiEidgD2\nBb7eg98jSZJmqKsz/Ih4G/AFYAS4L/BC4OnAgdUuZwBvjIhrgLXAqcB1wHk9qleSJM1At136uwAf\nBHYD1gPfBw7MzEsAMvO0iFgAnA3sCFwGHJyZd/auZEmS1K2uAj8zj57GPiuAFTOsR5IkzQHn0pck\nqQUMfEmSWsDAlySpBQx8SZJawMCXJKkFDHxJklrAwJckqQVmsniOJEmTGhkZYXR0tO4yNrHzzjuz\nePHiusuolYEvSeqJkZER9t57CRs23F53KZvYdtsFrFmzutWhb+BLknpidHS0CvtzgCV1l9NhNRs2\nHMHo6KiBL0lS7ywBXO68aRy0J0lSCxj4kiS1gIEvSVILGPiSJLWAgS9JUgsY+JIktUBjbstbvXp1\n3SVswpmZJEmDojGBf8QRR9RdwiacmUmSNCgaE/hwKvCcuovo4MxMkqTB0aDA3x1nZpIkaW40KPAl\nSapPE1f66+VYMgNfktR6TV3pr5djyQx8SVLrNXOlv96OJTPwJUm6x+Cu9OfEO5IktYCBL0lSCxj4\nkiS1gIEvSVILGPiSJLWAgS9JUgsY+JIktYCBL0lSC3QV+BHxDxHxrYi4NSLWRcSnI2KvSfY7JSJu\niIjbI+LCiNijdyVLkqRudXuGvz9wJrAv8Ezg3sCXIuI+YztExInAscAxwBOB24ALImKbnlQsSZK6\n1tXUupk5bsH6iPhb4BfAMuDyqvk44NTM/Fy1z5HAOuBQ4NxZ1itJkmZgttfwdwQSuBkgInYHdgUu\nHtshM28FrgD2m+XvkiRJMzTjwI+IAM4ALs/MH1bNu1IOANZN2H1dtU2SJNVgNqvl/QvwJ8BTelSL\nJEmaIzMK/Ij4Z+A5wP6ZeWPHppuAABYx/ix/EXDV1D/1XcDHJ7QNVQ9JktpteHiY4eHhcW3r16+f\n9vu7Dvwq7P8CeHpmjnRuy8xrI+Im4ADg+9X+O1BG9Z819U9+HfDCbsuRJKkVhoaGGBoafxK8atUq\nli1bNq33dxX4EfEvlFPu5wG3RcSiatP6zNxQPT8DeGNEXAOsBU4FrgPO6+Z3SZKk3un2DP8VlEF5\nX57Q/hLgQwCZeVpELADOpozivww4ODPvnF2pkiRpprq9D39ao/ozcwWwYgb1SJKkOeBc+pIktYCB\nL0lSCxj4kiS1gIEvSVILGPiSJLWAgS9JUgsY+JIktYCBL0lSCxj4kiS1gIEvSVILGPiSJLWAgS9J\nUgsY+JIktYCBL0lSCxj4kiS1gIEvSVILGPiSJLWAgS9JUgsY+JIktcDWdRcgqflGRkYYHR2tu4xx\ndt55ZxYvXlx3GVLfMPAlTWlkZIS9917Chg23113KONtuu4A1a1Yb+tI0GfiSpjQ6OlqF/TnAkrrL\nqaxmw4YjGB0dNfClaTLwJU3TEmBp3UVImiEH7UmS1AIGviRJLWDgS5LUAga+JEktYOBLktQCBr4k\nSS1g4EuS1AIGviRJLWDgS5LUAga+JEktYOBLktQCXQd+ROwfEZ+JiOsj4u6IeN4k+5wSETdExO0R\ncWFE7NGbciVJ0kzM5Ax/O+C7wN8BOXFjRJwIHAscAzwRuA24ICK2mUWdkiRpFrpeLS8zvwh8ESAi\nYpJdjgNOzczPVfscCawDDgXOnXmpkiRppnp6DT8idgd2BS4ea8vMW4ErgP16+bskSdL09XrQ3q6U\nbv51E9rXVdskSVINuu7SnzvvAj4+oW2oekiS1G7Dw8MMDw+Pa1u/fv2039/rwL8JCGAR48/yFwFX\nTf3W1wEv7HE5kiQNhqGhIYaGxp8Er1q1imXLlk3r/T3t0s/Maymhf8BYW0TsAOwLfL2Xv0uSJE1f\n12f4EbEdsAflTB7gYRGxD3BzZv4cOAN4Y0RcA6wFTgWuA87rScWSJKlrM+nSfzxwKWVwXlIuvgN8\nEDgqM0+LiAXA2cCOwGXAwZl5Zw/qlSRJMzCT+/C/whYuBWTmCmDFzEqSJEm95lz6kiS1gIEvSVIL\nGPiSJLWAgS9JUgsY+JIktYCBL0lSCxj4kiS1gIEvSVILGPiSJLWAgS9JUgsY+JIktYCBL0lSCxj4\nkiS1gIEvSVILGPiSJLWAgS9JUgsY+JIktYCBL0lSCxj4kiS1gIEvSVILGPiSJLWAgS9JUgsY+JIk\ntYCBL0lSCxj4kiS1wNZ1FyBJc2VkZITR0dG6y9jEzjvvzOLFi+suQy1j4EsaSCMjI+y99xI2bLi9\n7lI2se22C1izZrWhr3ll4EsaSKOjo1XYnwMsqbucDqvZsOEIRkdHDXzNKwNf0oBbAiytuwipdga+\nNE+8niypTga+NA+8niypbga+NA+8niypbgZ+S5155pk85SlPqbuMTUyne3l4eJihoaF5qqjXvJ6s\n6envz7maaM4CPyJeBRwP7Ap8D3h1Zn57rn6fpm9kZITXvnY5d999V92lbGI63cv+IVQb+DlXr81J\n4EfEXwPvAo4BvgUsBy6IiL0ys3mjllpmdHS0Cnu7lyWpLebqDH85cHZmfgggIl4BPBc4Cjhtjn6n\nutaf3ct33HEHq1atqruMcRzpLqnpeh74EXFvYBnwtrG2zMyIuAjYr9e/T+0yMjLCJZdcyrJly+ou\nZRxHuqvXPLBVr83FGf7OwFbAugnt64C9J9l/2/Kfr81BKbNxLQCrV6/e7B4bt50PbH6/+dWvdcN0\nay+XI14K7DY/ZW3RjWzY8H4uu+wyliyZ/BJJv/+bF02qvV/rhunUfuONN3LxxZc07sB2m2225VOf\n+iS77Tb5d6+f/82bWXs3dY9l6eZFZvaiqo0/MGI34Hpgv8y8oqP9ncDTMnO/Cfv/DfCRnhYhSVK7\nvDAzPzrVDnNxhj8K3AUsmtC+CLhpkv0vAF4IrAU2zEE9kiQNqm2Bh1KydEo9P8MHiIhvAldk5nHV\n6wBGgPdm5uk9/4WSJGlKczVK/93AByLiSjbelrcA+MAc/T5JkjSFOQn8zDw3InYGTqF05X8XOCgz\nfzkXv0+SJE1tTrr0JUlSs9yr7gIkSdLcM/AlSWoBA79FImKbiNg7IlwlUVJPRMSOEXF0RLw9Iu5f\ntS2NiD+uuzaN16jAj4gd665hSyJiRURs8u8WEQsjYriOmrYkIhZExPuB24H/ARZX7WdGxOtrLa4L\nEbFLROxfPXapu542iIitI+LhHiRqMhHxGODHwImU1VHH/oY/H3h7XXVpcrUFfkScWK2qN/b6XOBX\nEXF9ROxTV13T8FLg8oh42FhDRDwD+AHw8LqK2oK3A/sAz2D85EYXAX892RuaJCLuGxEfpszg+JXq\ncX1EnBMRC+utbjBFxH0i4mzgDmANGw8S3xMRf19rcQMsIoYi4isRMRIRD6naXhMRh9Rd22a8G/hA\nZu7J+L8t5wNPq6ckbU6dZ/ivAH4OEBHPAp4FHAx8AWjy5DyPAa4DvhsRL4uI04EvAR8GnlxrZZt3\nKHBsZl4OdN6W8T809yCl078D+wJ/TjmD2LF6/njg7Brr2qKIWBQRH46IGyLiDxFxV+ej7vqm8Fbg\nCZTvZecf8kuBxi3SHhGfmu6j7lo3JyKOAf4ZuISNa5IA/JYyl0kTPYHJv4PXA7vOcy1d6+Pv54zU\n2U23K1XgU/54n5uZX4qItcAVm31XzTLzFuDwiHgb5YP+B+DgzLy43sqm9ADgF5O0b8f4A4Cm+nPK\nPA6Xd7RdEBEvA75YU03T9QHK2fGpwI30x783lC7Zocz8RkR01nw1zTxIXN/xPIDDqrbvVG3LKAeK\njQ184Djg6Mz8dEQc39H+beCdNdW0Jb8DdpikfS+gH+Zd+QD9+f2ckToD/xbgwZTQfzbwxqo92Hhk\n20gR8WrKl3OY8ofkvRHxN5n5vXor26zvAM8Fzqxej32ojwa+UUtF3fkV4/+gj1lP+Rw12VOB/TPz\nu3UX0qVdmHztiwWU72ijZOZLxp5XC3WdC7wiM++q2rYC/gW4tZ4Kp+VhwGTr4W4Atp/nWqbrM8Cb\nI+Lw6nVGxGLKAcp/1VfWtPXr93NG6uzS/xTw0Yi4ENiJ0pUP8Djgmtqq2oKI+CJwEvDizHwhpd6v\nAt+MiBNqLW7z3gC8LSL+lXKQd1xEfAl4CfCPtVY2PW8B3h0R93QRVs9PpxyZN9nPaWBATsMq4Dkd\nr8cOEl9K8w8SjwL+aSzsAarn7662NdVaylibiQ6kOeu1TvQ6ysHIL4D7UMbXXAP8hv7429Kv388Z\nqfMMfznlA/5g4ITM/G3VvhvlSLyptgIek5k3AGTmHcArI+JzlGvNp9VZ3GQy8/KIeCzwesrgwgMp\nf9D3y8wf1Frc9LwS2AMYiYiRqm0xpTvxARHx8rEdM3NpDfVN5bXAOyLi5Zm5tu5iuvAG4PMR8QjK\n34lXRcQjgadXjybbGngEZbBhp0fQsDuTJjgD+OeIuDclhJZGxP+h9H6+otbKNiMz1wPPioinUsY3\nbQ+sysyL6q1s2vr1+zkjTq3bQxGxc2aO1l3HoImIk6a7b2aePJe1TEdE3ML4a4HbUULoduD3nftm\n5v3nsbSuRMSelODfh+oPOfD2Bl+6AiAi3g0cCbyNsngXlEGfrwc+nJn/t67atiQiXgysAB5SNa0D\nVmRmowen9pNB+X7ORG2BX32wRzPz89Xr04BjgB9SBgv9rJbCpqGaL+CvKIOXTs/MmyNiKbAuM6+v\nt7pNRcRkg2qgfOh/l5l3zmc9g676bE9LZn5wLmuZieqe+8OBizJzssGejVbNk3E8ZZzNblXzjcB7\ngHd1dvU3VfWd3X6sJ7FJIuI10903M987l7XMRL9/P2ejzsBfA7wyMy+JiP0o94Qvp4zI/kNmPr+W\nwragmmjiIsqAsYcCe2fm/0bEW4DFmXlknfVNJiLuZurRp9dRRquenJl3z0tRMxQR21LmDtgOuDAz\nf1JzSQMpIm4HljT5wHs6xg52M7PJg/UAiIg3AJdn5lcntC8AXpuZb6unsvEi4tpp7pqZ+bAt76b5\nUmfg3w48IjNHqlG1u2XmkdV1wi9n5gNqKWwLIuIiyjWqEyLiN8A+VeA/GfhoZj603go3FREvonRv\nfoCNXZxPBF5Mud96Z8oZ0elN+aMC93TN3jszX1293oZS/59Qut+2Bg7MzK/XV+XUIuI5wF2ZecGE\n9gOBrTLzC5O/s14R8VXKwLfP1F3LTFS9FM+g9MJ9NDN/ExEPBG7tGC/UKNWB+e+Bv+88M46IRcAN\nmdnou5f6UXWv/W4Te7IiYifgF4P2b17nAJbfUkbnQxlEdmH1fANltGdT9eNEEy8CXpeZb8rMz1aP\nN1FC/vDMfCvwGsp1zybp/FwAvJAyWG9P4H7AJ2j+SOB3bKb9XlNsa4IzgXdFxCsi4gkR8Sedj7qL\nm0o1Q90PgPOAsyjzUECZ/vWf6qprml4CnBQR/xZ9MJ1xRLy56oGY2H6fiHhzHTV1aXMj9P8IGLhL\nnXV+oC4E/j0irqJM0nB+1f5Iyuj9purHiSaeCrxqkvargP2q55dTTZ/aIIspYzrGHAh8cqybOSLe\nw8bPTVPtyaajxQF+RLnzoKk+Xv23846ZpPyBTJo9V8Z7KHNP7EOZw2HMp4F/q6Wi6buI8p38DHBp\nRBxWcz1bchLwPkqPW6cF1bZT5r2iaegYh5DA0RHR2euzFWVa4B/Ne2FzrM7AfxXl/uoHA3+ZmWNf\nzGWUCW2aqh8nmriOcv/0xIVyXsrG2Q53onmT2NzN+CPwJzH+vvtfU870m2w9ZUKVtRPa9wBum/dq\npm/PuguYhf2BJ2fmnRHjTuDWAk1ewS0BMvPHEfEkSg/WlTT0lrzK2AHgRPsAN89zLd0Ym6o4KP++\nnQM576R8Vpr87z4jtQV+Zv4aOHaS9mnfglWT1wGfZPxEE7tRJiNpavfy8cAnIuJgyjSdUOahXwL8\nZfX6CWw8q2uK1cAhlEl3Hkk547+0Y/tDKLctNdl5wBkRcVhm/hQgIvYA3kU5eGyksVr71L2YvAfi\nQZQJYZrqnqOTzPx1RDwbeC/lM9QoHbe2JfDjCdMvb0W5jfN9ddQ2HZm5O0BEXAo8v5oyfeDVfh9+\ndf1nMbBNZ3tmfr+eiqZnwkQTVzZ8Ln0i4qGUI9a9qqY1lLEI22fm1TWVNaWqO/NjlMsNjwS+nZmH\ndGx/J7B7Zh6+mR9Ruyir+X2RcoB1XdX8IOAyyh+aX9dV23RExF5M/v1s7KWUiPg4sD4zj6kG1j6G\ncrntPGCkcxreJomIlwLnZObvJrS/DHhaZr6onso2Vd3aFsB/UCav6Zz6+k5gbWY2fUbG1qlzlP4D\nKKPGnz3Z9qaNjqxuHdwpMz/X0fZi4GTK9ar/Bl498cvaRNWtSkOUaUYf37R/604RcQDlVs2bgDMz\n8/aObScBX8nML9dU3rRE6Vd+FqWb8w7g+xNvvWqaiNidconqsYy/dg807/vZKSIeBFxAqXlPyvX8\nPYFRSnD23dwCTRURTwe+npm/3+LODVTdCTSZpAwgvwY4LzObfHli2uoM/I9QumRfC3yZsrrVIso0\nkq8bm5CnKSLiC5TbBd9ZvX405fraByldz38PnJ2ZK2orcgsi4mmU6/Z/CdxAWc/gvzLz21O+UTNS\nTZH6RcoiLn01X0BEfIYSmC8DfkJZ+nknyvoFx2fmV2osb4uqEe5/zfhZAj9STYXdGBHxd8B/ZOaG\n6vnmZGb+63zVNZWI2GFsXoMpJvUCmj//QdWl/zjK5e2xwbV7Ua7p/wjYmxL+T83MH076Q/pInYF/\nI/AXmfmtiLiVcqb544h4HmVu/afWUthmVPUekpnfqV6/FXj6WJ3VnNcnZ2ajblmKssjM31KCfgeq\nVcQo8weNt3oSAAAQN0lEQVT0zQc4Iu5H+X9YUjWtpvyhbPSRd0T8kjKArN8CfxQ4IDO/V30/n5CZ\na6oel9MbuGbBPToDaZJte2RmYxbnioifA4/NzF9VzzcnM7MRd9F03rs+xaReQam5sT1BcM/Kp88A\nXtJxELOQsi7K5ZS7Oj4K3CczD6qrzl6pc5T+dmxco/0Wyr2yP6bcP9vEPyb3Y/wAsaezcYU/KIPh\nHjyvFW1BRHyWcnvJ5yk9KV/MzLsioq9Gn1Y9E59l/PrmrwbeFBGHNLx7/Bwmv0Oi6bZi41Kyo5SB\nqWuAaymL0DTZ5yPiWZm5obMxIvYGLqaMoWiEzHzwZM8b7s/YOAL/T+sspAdOAA7qPEDMzPURsQL4\nUma+JyJOAb5UV4G9VGfgr6F0l6wFvge8PCLWUs4+b6yvrM1aB+wO/Lya8W0p5T7TMfdlwsILDXAw\nZZTvv/bbGeYEZ1HuIHhlbrq++VnAo2usbUu2Bo6KiGdSLgGNuxUvm7uQy/9QBrtdS5nd8PiIuAN4\nedXWZL8FPhURz8vMPwBExBLgEkoPV1+IsibAHzXtMkRmfqWacOefmn5pZxruB+zC+Pk+oJyAjl2u\n+DUTBq32qzpn2nsPGxe2OJkSTiOUGd/eUFdRUzifsozi/sDbKRNNXNax/TFA025leirlQOTKiLgi\nIo6NiJ3rLmoG9mDCoie5cX3zJk9eA/AoyvXj31CuDT6u4/HYGuvakrex8YTgTZTavwH8BWVRmiZ7\nPrAQ+EgUj6KMExrOzMbVHhHPqaa/7mw7kXLgcmtEnB9lwa4mOYkyNqLfnQf8R0QcFhEPqh6HAe+n\nDMSGMg35j2ursJcysxEPykj3pcDOddeymfp2Br5KmQzmVuCwCdsvBt5ad52bqX07yoj8yym3zNxF\n+aN937prm2b9XwMOnaT9UOCbddc3SA/KJEGxmW27UOb/r73Oafx/7Ah8lzJ5zTrKuIPa69pMrZcA\nx3a8flL1HT2JsmrhGsq6BrXX2lHj3cAuddfRg/+P7SnX6X9X/ZvfVT3/f8B21T6PpYyzqL3e2T5q\nvw+/31QDOn6bE5bYjIj7V+2Nnn+5uo75Usr8+jtSVpx7Xr1VbSrKqoRjlgCnUeZ3/2bV9iTKbI2v\nz8ymTRjUt2LCYiLVPe2vycxGT3C0mdHiu1Gm8P4cHWMosmEjxyPiF5TryFdVr98FPCqrQWIR8Vxg\nZWbuNcWPmVfVYL1FmdnU6cS7EhHbUw52Af43G7rA0mzNa+BPcc/jJrK51zYHQnUN/BDgqIYG/tjo\n380tbjEms/kjgR9POVObbAKbRi0DXf2779oR+PesCFlvZVPbwmhx6JhLoGmfl2psxF6Z+fPq9RWU\n22VPq14/BPhhZm5XY5njVP/e65l62W0y8/7zU5GmY74H7T1umvvZ7TDHqh6K/2bjdaqm2b3uAnoh\nIl4AfIgyEcyBlNG+e1HmnPh0jaUNmn4eLX4D5c6Hn0fEdpQu5Nd1bL8/my5O0wQnMX6Gvb5T/Xu/\nHjiAcslq3Li2zHzYZO/rV/Ma+JnZz19KzaOsVsQbAG8AlmfmWdXZ8nGUUe5n08y7UcbmR5/Y1mhZ\njRavJtx5A2WOhuumfldjfJKy3sJbgOdSblfunJZ2GZOvuFi3j2X/z1r475RbrD9M+T42/rM+G3VO\nvLOQMgDo5gnt9wf+0LTrbJpf1QRMX8jM31fPNyszG7sITUTcBjwyM9dGxK+AZ2TmD8ZuE8vM3bbw\nI+ZV1VX7BcrAJSiXfS5h09sJG3UpolN1YPXozFxbdy3TUZ1l/htlCul1wNHZcbtbRHwFuCAz31ZT\niZuYONajX0XEr4HnZubX6q5lPtR5H/7HKLdETFxR6XDgecBz5r0iNcl/A7tSznamuuzQ9LXZb6Hc\nGglwPeU2vR9QBkwuqKuoKXxwwutzaqlidi6hnLWtrbmOacnM24C/mWL70+exnOna0tiafnELzV7G\nt6fqPMO/GdgvM9dMaH8E8LXM3KmWwqQeioiPAt/JzHdHxJsoMwSeR1lMZ1WTz5T7VTWT5EnAR5h8\nsqNG9ghVn4//7KNLEX0vIo6gzC3x4uxYmGtQ1Rn4twFPyswfTGh/NHBFZjbx7EfzKCZfofBIykRN\n29EHKxRWawBsm5k3VjOnnUBZiOYnwFuyJetwz6fqssTmNG6U/piIuJoy++glVBO/NP02334XEVcB\nD6f0WKxlwmyp2eA1I2aizsC/FLg6M189of0s4DGZuX8thakxNrNC4SrKssqNXqGwCvfjKWcP21Am\nZjo5GzZNqpolIp5AWezqBZTLVcOUAYhX1VnXoIqyxPZmZebJ81XLfKgz8J8CXERZdObiqvkA4AnA\ngZl52ebeq3bo1xUK4Z7u2ZMon/ENwEGUqV2PqrUw9YVqvY5DgZcAzwSupowo/1Bm/qbO2tS/ahu0\nl5lfi4gnUbo4DwfuAL4PvDT7e6EX9U7frVDY4Ujg7zLz/wFUi+d8PiKOzsypupzVA9XI96cz+WRH\n762lqO7cTZnmdeyzcjvl3vy3Vp+hT9ZW2YCp1in4K0rX/umZeXNELAXWZeb19VbXW/Me+JN0dV5C\nuQ3Frk5N1I8rFI5ZTMfBSWZeFBEJPBBwUNYciojHURa7WkAZ63EzZS2M2yl3fTQ28CNiH8pZ/d9Q\nwv7DwOsy80cREZR5HP6Zcu++ZqmawvsiygRCD6XcHnkzZQGmxZQD94FRx2p5/0hZies3lNuUXkNZ\n4lSaqB9XKByzNaUrv9PvgXvXUEvbrAQ+S+khuoOy7sJDKCP2j6+xrilVA8iupKwd8XfAgzPz7zPz\nR1BGG1Juk9ylvioHzruBD2Tmnoz/vp4PPK2ekubOvF/Dj4ifULpNxnV1Avexq1OdqqV8P0VZ5ve3\nlFtnPt2x/WLKann/WFOJmzXJBDYwySQ23pbXe9VkKvtm5prq+X6ZuToi9gU+mJmPqLnESUXEyZQB\nelPOMhkRW01cvEszExHrgaWZ+dPOdSOq9QvWZOa2NZfYU3Vcw7erU9OSmaPA0za3QiHwfygHAk00\ncQIb6M9JbPrR79l47fsXlL85qyndtk0d80FmTjlivGM/w753fgdMttLiXsBArATYqY7At6tTXcnM\nSRfomDgtc5Nk5kvqrqHFrqLc7fMT4CvAKVVv0Ysoo90bIyJOm+6+mXnCXNbSUp8B3hwRh1evMyIW\nA+8E/qu+suZGHV36dnVKmjPVcsT3zcxLI2IXymqFY5MdHZWZ36u1wA4RMd3bjzMzB+6act2q3sNP\nAo+nDAK+gTKl9zeBg6tpjwdGHYH/n9PZzzMkSdJ8qOaF2QfYnjLl9UU1lzQnapt4R5LmUnV2v3f1\n8keZOXDXZDU3IuJBwJsz85i6a+klA1/SQImI+wL/Qpmedmze/LuAjwOv2tyYkDpExLmUeUhurZ5v\nVmYePtV29U41H8Kqpq67MFN13IcvSXPp34F9KevL71g9/pxynfbsGuuazO8oc+aPPZ/qIc2KZ/iS\nBkq1EudBmXn5hPb9gS9m5nb1VKZ+4Rm+JPWHX1HuuZ9oPeByxGqt2hbPkaQ58hbg3RHxosy8CSAi\ndgVOB06ttbIpRMT9gBXAn1Kmzx13QpaZTqnbIxHxqS3ssuO8FDLPDHxJfa+ah77z+uSewEhEjFSv\nF1Ougz+A5l3HH/Nh4BHAf1IWjvJ669zZ0sDN9ZT5GwaKgS9pEPx33QX0wNOB/TPzu3UXMujaOs+L\ngS+p72XmyXXX0AM/piwZrnkWEXsADwe+mpl3RETkAI5od9CepIEVEdtHxA6dj7prmsKrgLdHxFMi\nYmFELOh81F3cIIqInapVN39MWRJ3t2rT+yPiXfVVNjcMfEkDJSJ2j4jPV7fnjY3MvwX4Nc0epf8L\nytSuXwVuBn4z4aHeW0lZvG0xcHtH+8eBZ9dS0RyyS1/SoDkHCOAo+mvw23D13yPpr7r72YGUORuu\ni4jO9p8AD6mnpLlj4EsaNPsAyzJzTd2FdOnRwNLM/FHdhbTIdow/sx9zfwZwdkO79CUNmm8DD667\niBlYBfxx3UW0zGWUHpUxGRH3Ak4ALq2npLnj1LqSBkpEPBx4H6Vr/2rKNdp7ZOb366hrSyLiLykT\n77wT+AGb1v3DGsoaaBHxKOBiysHWnwGfAR5JOcN/Smb+tMbyes7AlzRQIuJJwEeBh3Y0J+W6fjZ1\nfvSIuHuS5sbX3e8iYiFwLOVS0PaU8D8rM2+stbA5YOBLGigR8UNgNXAakwx+y8yf1VHXllQ9E5s1\naGebmn8GvqSBUt2Ot09mXlN3LWq+iNgWeAyTr1/wmVqKmiOO0pc0aC6hdM/2ZeBHxF6U+8LHzbqX\nmefXU9HgiohnU+bM33mSzQkM1GUUA1/SoPkssDIiHs3kg98aedYWEbsD/wU8lo5r9x27DFT4NMSZ\nwCeAUzJzXd3FzDW79CUNlM0MfhvT2MFvEfEZSsi/jDLxy5OBnSjL+h6fmV+psbyBFBG3Ao9ry/gI\nz/AlDZTM7Nf5RZ4MHJCZN0VEAndm5pcj4vWUKWCX1lveQPok8AzAwJekfhER5wNDmbm+ev164H2Z\n+evq9U7AZZn5JzWWOZWtgFur56OUhVzWANcCj6irqAF3LPCJiNifyS//vLeWquaIgS9pUBwE/FHH\n6zcA51IWzYHy927v+S6qC/9DGS1+LfAt4PiIuAN4edWm3huizKe/gXKm33mNOwEDX5IaKLbwuune\nBtynev4m4PPANygr/P11XUUNuLcCJwHvyMypxn4MBANfkmoUEQ8Dru287S4zfwLsFRG7AL/KzLtq\nK3CwbQN8vA1hDy6eI2lwJJsuKdsPtyH9BHjA2IuI+HhELALIzF8Y9nPqg7So98QzfEmDIoAPRMTY\nsqbbAu+rZt6D8df3m2TipYfnAP9QRyEttBVwQkQcBHyfTQft/d9aqpojBr6kQfHBCa/PmWSfD81H\nIeobjwauqp4/asK2fugd6ooT70hSjSLiLmDXzPxl9fo3wGMy05H56inP8CWpXlu6FAFAZj5/3ivT\nQDHwJale07kUoTkSEY8HDmfyBYsG6iDLwJekGmXmS+quoa0i4gWUcR0XUCbg+RKwF7AI+HSNpc0J\nb8uTJLXVG4DlmXkIcCdwHGUa43OBkToLmwsGviSprR5OmdEQSuBvl2Uk+0rgmNqqmiMGviSprW4B\n7ls9v56Nt+btCCyopaI55DV8SVJbfRV4FmWlvE8A74mIP6vaLq6zsLngffiSpFaKiPsD22bmDRFx\nL+AE4MmU6Y7fkpm31Fpgjxn4kiS1gF36kqRWiYi72fLUuZmZA5WRA/U/I0nSNBw2xbb9gNcwgIPa\n7dKXJLVeROwNvAM4BPgI8ObM/Fm9VfXWwB3BSJI0XRHxwIj4N8pI/a2Bx2bmiwct7MHAlyS1UEQs\njIh3AtcAjwQOyMxDMvPqmkubM17DlyS1SkScAJwI3AQMZeZ5NZc0L7yGL0lqlWqU/h3ARcBdm9vP\n1fIkSepvH2LLt+UNHM/wJUlqAQftSZLUAga+JEktYOBLktQCBr4kSS1g4EuS1AIGviRJLWDgS5LU\nAv8fAOmP7zzP+okAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x9094c10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.feature_selection import SelectKBest, f_classif\n",
    "import matplotlib.pyplot as plt\n",
    "predictors = [\"Pclass\", \"Sex\", \"Age\", \"SibSp\", \"Parch\", \"Fare\", \"Embarked\", \"FamilySize\", \"Title\", \"NameLength\"]\n",
    "\n",
    "# Perform feature selection\n",
    "selector = SelectKBest(f_classif, k=5)\n",
    "selector.fit(titanic[predictors], titanic[\"Survived\"])\n",
    "\n",
    "# Get the raw p-values for each feature, and transform from p-values into scores\n",
    "scores = -np.log10(selector.pvalues_)\n",
    "\n",
    "# Plot the scores.  See how \"Pclass\", \"Sex\", \"Title\", and \"Fare\" are the best?\n",
    "plt.bar(range(len(predictors)), scores)\n",
    "plt.xticks(range(len(predictors)), predictors, rotation='vertical')\n",
    "plt.show()\n",
    "\n",
    "# Pick only the four best features.\n",
    "predictors = [\"Pclass\", \"Sex\", \"Fare\", \"Title\"]\n",
    "\n",
    "alg = RandomForestClassifier(random_state=1, n_estimators=50, min_samples_split=8, min_samples_leaf=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.821548821549\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "e:\\python27\\lib\\site-packages\\ipykernel\\__main__.py:37: FutureWarning: in the future, boolean array-likes will be handled as a boolean array index\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "import numpy as np\n",
    "\n",
    "# The algorithms we want to ensemble.\n",
    "# We're using the more linear predictors for the logistic regression, and everything with the gradient boosting classifier.\n",
    "algorithms = [\n",
    "    [GradientBoostingClassifier(random_state=1, n_estimators=25, max_depth=3), [\"Pclass\", \"Sex\", \"Age\", \"Fare\", \"Embarked\", \"FamilySize\", \"Title\",]],\n",
    "    [LogisticRegression(random_state=1), [\"Pclass\", \"Sex\", \"Fare\", \"FamilySize\", \"Title\", \"Age\", \"Embarked\"]]\n",
    "]\n",
    "\n",
    "# Initialize the cross validation folds\n",
    "kf = KFold(titanic.shape[0], n_folds=3, random_state=1)\n",
    "\n",
    "predictions = []\n",
    "for train, test in kf:\n",
    "    train_target = titanic[\"Survived\"].iloc[train]\n",
    "    full_test_predictions = []\n",
    "    # Make predictions for each algorithm on each fold\n",
    "    for alg, predictors in algorithms:\n",
    "        # Fit the algorithm on the training data.\n",
    "        alg.fit(titanic[predictors].iloc[train,:], train_target)\n",
    "        # Select and predict on the test fold.  \n",
    "        # The .astype(float) is necessary to convert the dataframe to all floats and avoid an sklearn error.\n",
    "        test_predictions = alg.predict_proba(titanic[predictors].iloc[test,:].astype(float))[:,1]\n",
    "        full_test_predictions.append(test_predictions)\n",
    "    # Use a simple ensembling scheme -- just average the predictions to get the final classification.\n",
    "    test_predictions = (full_test_predictions[0] + full_test_predictions[1]) / 2\n",
    "    # Any value over .5 is assumed to be a 1 prediction, and below .5 is a 0 prediction.\n",
    "    test_predictions[test_predictions <= .5] = 0\n",
    "    test_predictions[test_predictions > .5] = 1\n",
    "    predictions.append(test_predictions)\n",
    "\n",
    "# Put all the predictions together into one array.\n",
    "predictions = np.concatenate(predictions, axis=0)\n",
    "\n",
    "# Compute accuracy by comparing to the training data.\n",
    "accuracy = sum(predictions[predictions == titanic[\"Survived\"]]) / len(predictions)\n",
    "print(accuracy)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1     240\n",
      "2      79\n",
      "3      72\n",
      "4      21\n",
      "7       2\n",
      "6       2\n",
      "10      1\n",
      "5       1\n",
      "Name: Title, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "titles = titanic_test[\"Name\"].apply(get_title)\n",
    "# We're adding the Dona title to the mapping, because it's in the test set, but not the training set\n",
    "title_mapping = {\"Mr\": 1, \"Miss\": 2, \"Mrs\": 3, \"Master\": 4, \"Dr\": 5, \"Rev\": 6, \"Major\": 7, \"Col\": 7, \"Mlle\": 8, \"Mme\": 8, \"Don\": 9, \"Lady\": 10, \"Countess\": 10, \"Jonkheer\": 10, \"Sir\": 9, \"Capt\": 7, \"Ms\": 2, \"Dona\": 10}\n",
    "for k,v in title_mapping.items():\n",
    "    titles[titles == k] = v\n",
    "titanic_test[\"Title\"] = titles\n",
    "# Check the counts of each unique title.\n",
    "print(pandas.value_counts(titanic_test[\"Title\"]))\n",
    "\n",
    "# Now, we add the family size column.\n",
    "titanic_test[\"FamilySize\"] = titanic_test[\"SibSp\"] + titanic_test[\"Parch\"]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.11682912,  0.47835566,  0.12614824,  0.13098157,  0.52105874,\n",
       "        0.1435209 ,  0.64085331,  0.18003152,  0.67801353,  0.12111118,\n",
       "        0.12105181,  0.20902118,  0.91068381,  0.1089127 ,  0.89142102,\n",
       "        0.87713474,  0.16349859,  0.13907791,  0.54103238,  0.55661006,\n",
       "        0.22420875,  0.5372079 ,  0.90572223,  0.38890588,  0.88384752,\n",
       "        0.10357315,  0.90909441,  0.13746454,  0.31046249,  0.12665718,\n",
       "        0.11663767,  0.18274855,  0.55220994,  0.49648575,  0.42415297,\n",
       "        0.14191051,  0.50973638,  0.52452209,  0.13270506,  0.28366691,\n",
       "        0.11145281,  0.46618807,  0.09996501,  0.83420617,  0.89959119,\n",
       "        0.14983417,  0.31593419,  0.13789623,  0.89104185,  0.54189565,\n",
       "        0.35666363,  0.17718135,  0.8307195 ,  0.87995521,  0.1755907 ,\n",
       "        0.13741805,  0.10667279,  0.1234385 ,  0.12099736,  0.91285169,\n",
       "        0.13099159,  0.15341948,  0.12993967,  0.66573206,  0.66343836,\n",
       "        0.87272604,  0.67238712,  0.288265  ,  0.35236574,  0.85565507,\n",
       "        0.6622414 ,  0.12701993,  0.55390065,  0.36740462,  0.91110312,\n",
       "        0.41201902,  0.13014004,  0.83671279,  0.15614414,  0.6622414 ,\n",
       "        0.68129213,  0.20605719,  0.20382623,  0.12105181,  0.18486634,\n",
       "        0.13130212,  0.65680539,  0.53029858,  0.65489631,  0.79881212,\n",
       "        0.53764546,  0.12104028,  0.8913725 ,  0.13014004,  0.28406245,\n",
       "        0.12345367,  0.86792484,  0.14666337,  0.58599461,  0.12260781,\n",
       "        0.90433464,  0.14730817,  0.13789623,  0.12262433,  0.62257491,\n",
       "        0.13155874,  0.14607753,  0.13789623,  0.13020336,  0.17473033,\n",
       "        0.14286392,  0.65490316,  0.89528117,  0.67146758,  0.88346017,\n",
       "        0.13992078,  0.11805064,  0.69612515,  0.36668939,  0.86241698,\n",
       "        0.87649291,  0.12609327,  0.90276371,  0.12099027,  0.13789623,\n",
       "        0.56971935,  0.12608181,  0.63733743,  0.13339996,  0.13340574,\n",
       "        0.12723637,  0.51609607,  0.23921874,  0.10791695,  0.09896737,\n",
       "        0.12431124,  0.13346495,  0.16214099,  0.52029433,  0.12232635,\n",
       "        0.20712059,  0.90529649,  0.19747926,  0.16153716,  0.42927593,\n",
       "        0.10487176,  0.33642492,  0.13518414,  0.46618807,  0.34478758,\n",
       "        0.91431377,  0.13214999,  0.10690998,  0.48983645,  0.11274825,\n",
       "        0.12427868,  0.9107016 ,  0.57991631,  0.42927593,  0.51274048,\n",
       "        0.65489239,  0.57884522,  0.82113381,  0.12096648,  0.28979611,\n",
       "        0.58587108,  0.30130471,  0.14606803,  0.9025041 ,  0.52257377,\n",
       "        0.12101884,  0.13299498,  0.12418534,  0.13207486,  0.1319655 ,\n",
       "        0.8729358 ,  0.87633414,  0.29670328,  0.83389526,  0.85558679,\n",
       "        0.15614414,  0.33352246,  0.90219082,  0.13789623,  0.91718918,\n",
       "        0.13603003,  0.85482389,  0.12241402,  0.14217314,  0.13560687,\n",
       "        0.1348803 ,  0.25547183,  0.49950989,  0.12729496,  0.71980831,\n",
       "        0.10795469,  0.85516508,  0.58990449,  0.16645668,  0.53980354,\n",
       "        0.64867969,  0.66329187,  0.60981573,  0.87333314,  0.16322638,\n",
       "        0.25696649,  0.63083524,  0.16482591,  0.88984707,  0.12346408,\n",
       "        0.12849653,  0.12097124,  0.24675029,  0.80199995,  0.41248342,\n",
       "        0.29768148,  0.65492663,  0.21860346,  0.90027407,  0.13014004,\n",
       "        0.8137002 ,  0.13611142,  0.84275393,  0.12700828,  0.87789288,\n",
       "        0.59807994,  0.12518087,  0.65489631,  0.11487493,  0.1441311 ,\n",
       "        0.25075165,  0.89266286,  0.11622683,  0.1379133 ,  0.34224639,\n",
       "        0.12796773,  0.19365861,  0.14018901,  0.80948189,  0.89790832,\n",
       "        0.87598967,  0.82598174,  0.33036559,  0.12105101,  0.33258156,\n",
       "        0.28710745,  0.8790295 ,  0.16058987,  0.86241698,  0.59133092,\n",
       "        0.74586492,  0.15434326,  0.39647431,  0.13354268,  0.12701864,\n",
       "        0.12101884,  0.13789623,  0.13014004,  0.83005787,  0.12700585,\n",
       "        0.10894954,  0.12701508,  0.85003763,  0.64929875,  0.16619539,\n",
       "        0.12105181,  0.21821016,  0.12101884,  0.50973638,  0.14016481,\n",
       "        0.34495861,  0.13789623,  0.91564   ,  0.6332826 ,  0.13207439,\n",
       "        0.85713531,  0.15861636,  0.12500116,  0.14267175,  0.16811853,\n",
       "        0.52045075,  0.66231856,  0.65489631,  0.64136782,  0.71198852,\n",
       "        0.10601085,  0.12099027,  0.3627808 ,  0.13207486,  0.13014004,\n",
       "        0.33304456,  0.59319589,  0.13207486,  0.50584352,  0.12081676,\n",
       "        0.12263655,  0.77903176,  0.12665718,  0.33024483,  0.12028976,\n",
       "        0.11813957,  0.17547887,  0.1216941 ,  0.13347145,  0.65489631,\n",
       "        0.82133626,  0.33497525,  0.67696014,  0.20916505,  0.42575111,\n",
       "        0.13912869,  0.13799529,  0.12102122,  0.61904744,  0.90111957,\n",
       "        0.67393647,  0.23919457,  0.17328806,  0.12182854,  0.18522951,\n",
       "        0.12262433,  0.13491478,  0.16214099,  0.45541306,  0.90601333,\n",
       "        0.12509883,  0.86563776,  0.34598576,  0.14469719,  0.17034218,\n",
       "        0.82147627,  0.32823572,  0.13207439,  0.64322911,  0.12183262,\n",
       "        0.25111398,  0.15333425,  0.09370087,  0.20950803,  0.35411806,\n",
       "        0.17507148,  0.118123  ,  0.1469565 ,  0.91556464,  0.33657652,\n",
       "        0.618368  ,  0.16214099,  0.62462682,  0.1654289 ,  0.85157883,\n",
       "        0.89603825,  0.16322638,  0.24472808,  0.16066609,  0.70031025,\n",
       "        0.15642457,  0.85672648,  0.12105022,  0.13789623,  0.57255235,\n",
       "        0.10418822,  0.87672475,  0.86918839,  0.13098157,  0.91914163,\n",
       "        0.15715004,  0.1313025 ,  0.53322127,  0.89562968,  0.17356053,\n",
       "        0.15319843,  0.90891499,  0.16307942,  0.13130575,  0.87654859,\n",
       "        0.90969185,  0.48853359,  0.17002326,  0.19866966,  0.13510974,\n",
       "        0.13789623,  0.14010265,  0.54133852,  0.5949924 ,  0.15905635,\n",
       "        0.83276875,  0.12430276,  0.12019388,  0.14606637,  0.18789784,\n",
       "        0.38579307,  0.87750065,  0.56459193,  0.12807839,  0.10318132,\n",
       "        0.91169572,  0.14231524,  0.88773179,  0.12607946,  0.12971145,\n",
       "        0.90753797,  0.12635163,  0.90891637,  0.35988713,  0.30442425,\n",
       "        0.18966803,  0.1501521 ,  0.26822399,  0.65488945,  0.64585313,\n",
       "        0.65489631,  0.90711865,  0.56933478,  0.13014004,  0.86010063,\n",
       "        0.10126674,  0.13014004,  0.41850311])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictors = [\"Pclass\", \"Sex\", \"Age\", \"Fare\", \"Embarked\", \"FamilySize\", \"Title\"]\n",
    "\n",
    "algorithms = [\n",
    "    [GradientBoostingClassifier(random_state=1, n_estimators=25, max_depth=3), predictors],\n",
    "    [LogisticRegression(random_state=1), [\"Pclass\", \"Sex\", \"Fare\", \"FamilySize\", \"Title\", \"Age\", \"Embarked\"]]\n",
    "]\n",
    "\n",
    "full_predictions = []\n",
    "for alg, predictors in algorithms:\n",
    "    # Fit the algorithm using the full training data.\n",
    "    alg.fit(titanic[predictors], titanic[\"Survived\"])\n",
    "    # Predict using the test dataset.  We have to convert all the columns to floats to avoid an error.\n",
    "    predictions = alg.predict_proba(titanic_test[predictors].astype(float))[:,1]\n",
    "    full_predictions.append(predictions)\n",
    "\n",
    "# The gradient boosting classifier generates better predictions, so we weight it higher.\n",
    "predictions = (full_predictions[0] * 3 + full_predictions[1]) / 4\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
